'''
Created on April 12, 2011

@author: posttool

This module provides all logic and web services for Project, Dashboard, Widgets, etc.
It integrates with all other modules: authorization, queue, share, platform, widgets
'''
import logging
import datetime
import cStringIO
import csv
import traceback

from google.appengine.ext import db
from django.utils import simplejson
from impact import platform, authorization, widget, widgets, queue

from platform import ImpactException
from widget import WidgetDefinition
from impact.authorization import IntegrationPointInstance
from impact.widget import WidgetRegistry

config=None
def init(app_config):
	global config
	config 		= app_config['dashboard']

def get_url_mappings():
	return [
		('/api/dashboard/(.*)', DashboardApi),
		('/dashboard', DashboardPage),
		('/dashboard/authorize/return/(.*)', DashboardAuthReturnPage),
		('/dashboard/authorize/(.*)', DashboardAuthPage),
		('/dashboard-embed', DashboardEmbed),
		('/dashboard-cron/(.*)', DashboardCron),
		('/api/connections/(.*)', ConnectionsApi),
		('/connections', ConnectionsPage),
		('/connections/authorize/return/(.*)', ConnectionsAuthReturnPage),
		('/connections/authorize', ConnectionsAuthPage),
		]

## TEMPLATED PAGE

class DashboardPage(platform.TemplateHandler):
	'''
	This is the main dashboard page. It prepares the widget store data as json which
	is used to bootstrap the UI (via django template and javascript).
	'''
	def __init__(self):
		self.template = "templates/dashboard.html"

	def prepare_page(self,uctx,inputs,pagedata):
		root, registry = widget.get_widget_root()
		pagedata['widget_categories'] = root.categories
		pagedata['widget_registry_json'] = platform.util.query_to_json(registry)
		pagedata['project'] = _get_project(uctx)


class DashboardAuthPage(platform.ImpactBaseHandler):
	'''
	When the UI needs a Connection, it interacts with the DashboardAuthPage.
	A WidgetInstance key is required from which the Connection type & scope is derived.
	This page then redirects to the authorization service with type & scope as well as
	the parent Project, eventually arriving at the DashboardAuthReturnPage.
	This page must be requested in a pop up window.
	'''
	def process(self,uctx,path):
		args = path.split('/')
		wi = WidgetInstance.get(db.Key(encoded=args[0]))
		if wi == None:
			raise ImpactException('There is no such Widget Instance')
		guard_permissions(uctx, wi.dashboard.project)
		idx 	     = int(args[1])
		connection_descriptors	 = wi.get_required_connections()
		if idx > (len(connection_descriptors)-1):
			raise ImpactException('No connection at idx '+str(idx))
		connection_descriptor = connection_descriptors[idx]
		type 	= connection_descriptor['type']
		scopes 	= connection_descriptor['scopes']
		return_to = '/dashboard/authorize/return/'+args[0]+'/'+args[1]
		redirect_url = '/authorize/'+type+'?r='+return_to+'&p='+str(_get_project(uctx).key())
		if len(scopes) > 0:
			scopes_str = ','.join(scopes)
			redirect_url= redirect_url + '&scopes='+scopes_str
		self.redirect(redirect_url)

class DashboardAuthReturnPage(platform.ImpactBaseHandler):
	'''
	The Connection (aka IntegrationPointInstance) is added to the WidgetInstance that initiated
	the request for it... On completion, this page writes a script tag to the response that communicates
	with the window.opener, calling the function 'authorization_complete' with the json representation
	of the Connection. This requires the page that opened the pop up also define the authorization_complete
	function. There is no callback if the authorization process is canceled.
	'''
	def process(self,uctx,path):
		args=path.split('/')
		wi = WidgetInstance.get(db.Key(encoded=args[0]))
		if wi == None:
			raise ImpactException('There is no such Widget Instance')
		idx  = int(args[1])
		connection_descriptors	 = wi.get_required_connections()
		if idx > (len(connection_descriptors)-1):
			raise ImpactException('No connection at index '+str(idx))
		connection_descriptor = connection_descriptors[idx]
		ip = authorization.IntegrationPointInstance.get(db.Key(encoded=self.request.get("ip")))
		#if ip.parent() != wi.dashboard.project: #TODO test
		#	raise ImpactException('Bad Connection')
		wi.connections[connection_descriptor['name']] = ip
		wi.state = WIDGET_STATE_INIT;
		wi.save(wi.user)
		self.response.out.write('<script>window.opener.authorization_complete('+
							platform.util.entity_to_json(ip)+');this.close()</script>')




class DashboardEmbed(platform.TemplateHandler):

	def __init__(self):
		self.template = "templates/dashboard-embed.html"

	def prepare_page(self,uctx,inputs,pagedata):
		dash = Dashboard.get(db.Key(encoded=inputs['p']))
		if dash == None:
			raise Exception('There is no such dashboard')
		if not dash.public:
			raise Exception('The dashboard is not visible')
		widgs = dash.widgets
		pagedata['dashboard_json'] = simplejson.dumps(dash.to_dict())
		pagedata['widgets_json'] = platform.util.query_to_json(widgs)


#########################
## SERVICE

class DashboardCron(platform.ServiceHandler):
	def run(self,uctx,args,params):
		process_all_widget_instances()
		return {'message':'ok'}


class DashboardApi(platform.ServiceHandler):
	'''
	Services related to managing dashboards and widgets.
	'''
	def list(self,uctx,args,params):
		"""returns a list of dashboards (for a user)"""
		project = _get_project(uctx)
		query = db.Query(Dashboard)
		query.ancestor(project)
		query.order('name')
		return platform.util.query_to_dict_list(query)

	def add(self,uctx,args,params):
		"""adds a dashboard. requires the dashboard name."""
		if len(args)!=1:
			raise Exception('Please provide a name for the dashboard.')
		if _dashboard_name_exists(uctx,args[0]):
			raise Exception('There is an existing dashboard named ['+args[0]+']')
		p = _get_project(uctx)
		dash = Dashboard(parent=p, name=args[0])
		dash.save(uctx.user)
		return dash.to_dict()

	def duplicate(self,uctx,args,params):
		"""duplicates a dashboard. requires the dashboard key and new name."""
		if len(args)!=2:
			raise Exception('Please provide a name for the dashboard.')
		if _dashboard_name_exists(uctx,args[1]):
			raise Exception('There is an existing dashboard named ['+args[1]+']')
		orig_dash = Dashboard.get(db.Key(encoded=args[0]))
		if orig_dash is None:
			raise Exception('No such dashboard.')
		guard_permissions(uctx, orig_dash.project)

		#need to get rid of transaction or maybe make widget a
		# string 'widget_reg_class"
		def txn():
			dash = Dashboard(parent=orig_dash.project, name=args[1])
			dash.save(uctx.user)
			orig_widgs = orig_dash.widgets
			for orig_wi in orig_widgs:
				wi 		= orig_wi.clone(dash)
				wi.name = args[1]
				wi.save(uctx.user)
				q = db.Query(WidgetData)
				q.ancestor(orig_wi)
				for orig_data in q:
					wd = WidgetData(parent=wi,date=orig_data.date,data=orig_data.data)
					wd.save(uctx.user)
			return dash.to_dict()

		return db.run_in_transaction(txn)

	def delete(self,uctx,args,params):
		"""removes a dashboard. requires the dashboard key."""
		if len(args)!=1:
			raise Exception('Requires dashboard key.')
		dash = Dashboard.get(db.Key(encoded=args[0]))
		if dash == None:
			raise Exception('There is no such dashboard')
		guard_permissions(uctx, dash.project)
		dash.delete();
		return dash.to_dict()

	def tags(self,uctx,args,param):
		"""requires the dash id and tags"""
		if len(args)!=2:
			raise Exception('Requires dashboard key and tags.')
		dash = Dashboard.get(db.Key(encoded=args[0]))
		if dash == None:
			raise Exception('There is no such dashboard')
		guard_permissions(uctx, dash.project)
		tags = []
		for t in args[1].split(','):
			tags.append(t.strip())
		dash.tags = tags
		dash.save(uctx.user)
		return dash.to_dict()

	def public(self,uctx,args,param):
		"""requires the dash id and boolean"""
		if len(args)!=2:
			raise Exception('Requires dashboard key and boolean.')
		dash = Dashboard.get(db.Key(encoded=args[0]))
		if dash == None:
			raise Exception('There is no such dashboard')
		guard_permissions(uctx, dash.project)
		dash.public = args[1] == "yes" or args[1] == "True" or args[1] == "true"
		dash.save(uctx.user)
		return dash.to_dict()

	def export(self,uctx,args,param):
		"""requires the dash id and boolean"""
		if len(args)!=1:
			raise Exception('Requires dashboard key')
		dash = Dashboard.get(db.Key(encoded=args[0]))
		guard_permissions(uctx, dash.project)
		queue = cStringIO.StringIO()
		writer = csv.writer(queue, dialect=csv.excel)
		for wi in dash.widgets:
			writer.writerow([wi.widget.key().name(),wi.rect])
			query = db.Query(WidgetData)
			query.ancestor(wi)
			query.order('-date')
			results = query.fetch(1000)##TODO page apply
			for r in results:
				if r.data is not None and r.data.has_key('value'):
					writer.writerow([r.date,r.data['value']])
				elif r.data is not None:
					writer.writerow([r.date,str(r.data)])
		return queue.getvalue()

	def get_widgets(self,uctx,args,param):
		"""get widgets for a dashboard. requires dashboard key"""
		dash = Dashboard.get(db.Key(encoded=args[0]))
		if dash == None:
			raise Exception('There is no such dashboard')
		guard_permissions(uctx,dash.project)
		return platform.util.query_to_dict_list(dash.widgets)

	def add_widget(self,uctx,args,param):
		"""get widgets for a dashboard. requires dashboard key, widget type, x, y"""
		dash = Dashboard.get(db.Key(encoded=args[0]))
		if dash == None:
			raise Exception('There is no such dashboard')
		guard_permissions(uctx, dash.project)
		widg = widget.WidgetRegistry.get_by_key_name(args[1])
		if widg == None:
			raise Exception('There is no such widget')

		if widg.definition()._is_correlation():
			wi = create_correlation_widget_instance(dash,widg,[int(args[2]),int(args[3]),1,1])
		else:
			wi = create_widget_instance(dash,widg,[int(args[2]),int(args[3]),1,1])
		return enter_widget_state_machine(wi).to_dict()
	
	def convert_to_correlation(self,uctx,args,param):
		"""handles the vs case"""
		wi = WidgetInstance.get(db.Key(encoded=args[0]))
		if wi == None:
			raise Exception('There is no such widget instance')
		guard_permissions(uctx, wi.dashboard.project)
		widget = WidgetRegistry.get_by_key_name('impact_correlation')
		corr = create_correlation_widget_instance(wi.dashboard,widget,wi.rect)
		def txn():
			wi.type = WIDGET_TYPE_CORRELATION_MEMBER
			wi.save()
			corr.children.append(wi.key())
			corr.save()
		db.run_in_transaction(txn)
		return enter_widget_state_machine(corr).to_dict()
		

	def add_widget_to_correlation(self,uctx,args,param):
		"""get widgets for a dashboard. requires dashboard key, widget type, x, y"""
		##TODO really needs a transaction
		corr = WidgetInstance.get(db.Key(encoded=args[0]))
		if corr == None:
			raise Exception('There is no such correlation')
		guard_permissions(uctx, corr.dashboard.project)
		if not corr.definition.is_correlation:
			raise Exception('Widget with key '+args[0]+' is not a correlation')
		wi = WidgetInstance.get(db.Key(encoded=args[1]))
		if wi == None:
			raise Exception('There is no such widget')
		if wi.definition()._is_correlation():
			raise Exception('Cannot add correlation to a correlation')

		def txn():
			wi.type = WIDGET_TYPE_CORRELATION_MEMBER
			wi.save()
			corr.children.append(wi.key())
			corr.save()
		db.run_in_transaction(txn)

		return enter_widget_state_machine(corr).to_dict()

	def remove_widget_from_correlation(self,uctx,args,params):
		corr = WidgetInstance.get(db.Key(encoded=args[0]))
		if corr == None:
			raise Exception('There is no such correlation')
		guard_permissions(uctx, corr.dashboard.project)
		if not corr.definition.is_correlation:
			raise Exception('Widget with key '+args[0]+' is not a correlation')
		wi = WidgetInstance.get(db.Key(encoded=args[1]))
		if wi == None:
			raise Exception('There is no such widget')
		if  wi.type != WIDGET_TYPE_CORRELATION_MEMBER:
			raise Exception('Argument error')
		wi.type = WIDGET_TYPE_NORMAL
		wi.rect[0] = int(params['x'])
		wi.rect[1] = int(params['y'])
		wi.rect[2] = int(params['w'])
		wi.rect[3] = int(params['h'])

		def txn():
			corr.children.remove(wi.key())
			wi.save()
			corr.save()
		db.run_in_transaction(txn)
		corr.state = WIDGET_STATE_CORRELATION_REQUIRES_MEMBERS;
		corr = enter_widget_state_machine(corr).to_dict();
		return {'corr':corr,'wi':wi.to_dict()}

	def update_widget_rect(self,uctx,args,param):
		"""updates a widgets rect. requires widget id. optional params: x, y, w, h"""
		widg = WidgetInstance.get(db.Key(encoded=args[0]))
		if widg == None:
			raise Exception('There is no such widget')
		guard_permissions(uctx, widg.dashboard.project)
		if param.has_key('x'):
			widg.rect[0] = int(param['x'])
		if param.has_key('y'):
			widg.rect[1] = int(param['y'])
		if param.has_key('w'):
			widg.rect[2] = int(param['w'])
		if param.has_key('h'):
			widg.rect[3] = int(param['h'])
		widg.save(uctx.user)
		return widg.to_dict()

	def delete_widget(self,uctx,args,param):
		"""deletes a widget. requires widget id."""
		widg = WidgetInstance.get(db.Key(encoded=args[0]))
		if widg == None:
			raise Exception('There is no such widget')
		guard_permissions(uctx, widg.dashboard.project)
		widg.delete()
		return widg.to_dict()

	def get_data_for_widget(self,uctx,args,param):
		"""requires the widget instance id"""
		wi = WidgetInstance.get(db.Key(encoded=args[0]))
		if wi == None :
			raise Exception('There is no such widget')
		guard_permissions(uctx, wi.dashboard.project)
		if wi.is_live_data_widget():
			try:
				data = {'values':[{'data':wi.get_data()}]}
				return data
			except:
				logging.info('can t get data for '+str(wi))
				traceback.print_exc()


		editor = wi.get_props_editor();
		if editor == WidgetDefinition.PROPS_EDITOR_TYPE_CORRELATION:
			return get_data_for_trend_props_correlation_widget(wi);
		return get_data_for_trend_props_widget(wi);

	def update_props_for_widget(self,uctx,args,param):
		"""requires the widget instance id and param.props"""
		wi = WidgetInstance.get(db.Key(encoded=args[0]))
		if wi == None:
			raise Exception('There is no such widget')
		guard_permissions(uctx, wi.dashboard.project)
		logging.info(" setting props to "+str(param))
		wi.props = param
		wi.save()
		return wi.to_dict()

	def set_connection_for_widget(self,uctx,args,param):
		"""requires the widget instance id and the integration point instance id"""
		wi = WidgetInstance.get(db.Key(encoded=args[0]))
		if wi == None:
			raise ImpactException('There is no such widget')
		guard_permissions(uctx, wi.dashboard.project)
		idx = int(args[1])
		connection_descriptors	 = wi.get_required_connections()
		if idx > (len(connection_descriptors)-1):
			raise ImpactException('No connection at idx '+str(idx))
		conn = authorization.IntegrationPointInstance.get(db.Key(encoded=args[2]))
		if conn == None:
			raise ImpactException('There is no such connection')
		guard_permissions(uctx, wi.dashboard.project)
		connection_descriptor = connection_descriptors[idx]
		wi.connections[connection_descriptor['name']] = conn
		wi.state = WIDGET_STATE_INIT
		return enter_widget_state_machine(wi).to_dict()

	def get_selector_data_for_widget(self,uctx,args,param):
		"""requires the widget instance id,selector name, and selector value"""
		if len(args) < 2:
			raise ImpactException('Missing required args')
		widget_instance_key = args[0]
		name 		   		= args[1]
		wi = WidgetInstance.get(db.Key(encoded=widget_instance_key))
		if wi == None:
			raise Exception('There is no such widget')
		guard_permissions(uctx, wi.dashboard.project)
		#default to first
		selector_display_values, selector_values  = wi.get_selector_data(name)
		if not selector_values:
			selector_values = selector_display_values
		return {'selector_name':name,
				'display_values':selector_display_values,
				'values': selector_values }


	def set_selector_value_for_widget(self,uctx,args,param):
		"""requires the widget instance id,selector name, and selector value"""
		if len(args) < 3:
			raise ImpactException('Missing required args')
		widget_instance_key = args[0]
		name 		   		= args[1]
		value		   		= args[2]
		display_value  		= args[3]
		wi = WidgetInstance.get(db.Key(encoded=widget_instance_key))
		if wi == None:
			raise ImpactException('There is no such widget')
		guard_permissions(uctx, wi.dashboard.project)
		wi.selector_values[name] = value
		wi.selector_display_values[name] = display_value
		#this validates the choice
		wi.state 	  = WIDGET_STATE_REQUIRES_SERVICE_SELECTION
		return enter_widget_state_machine(wi).to_dict()

	def refresh_widget_data(self,uctx,args,param):
		"""requires the widget instance id,selector name, and selector value"""
		if len(args) < 1:
			raise ImpactException('Missing required args')

		widget_instance_key = args[0]
		wi = WidgetInstance.get(db.Key(encoded=widget_instance_key))
		if wi == None:
			raise ImpactException('There is no such widget')
		guard_permissions(uctx, wi.dashboard.project)
		return enter_widget_state_machine(wi).to_dict()

	def sync_widget(self,uctx,args,params):
		if len(args) < 1:
			raise ImpactException('Missing required args')
		widget_instance_key = args[0]
		wi = WidgetInstance.get(db.Key(encoded=widget_instance_key))
		guard_permissions(uctx, wi.dashboard.project)
		return enter_widget_state_machine(wi).to_dict()

	def get_props_editor(self,uctx,args,param):
		widget_instance_key = args[0]
		wi = WidgetInstance.get(db.Key(encoded=widget_instance_key))
		if wi == None:
			raise ImpactException('There is no such widget')
		guard_permissions(uctx, wi.dashboard.project)
		props_template = wi.get_props_editor_template()
		expanded_template = platform.util.render_django_template(props_template, {'wid':str(wi.key()),'props':wi.props})
		return expanded_template

#	def get_connections(self,uctx,args,param):
#		query = db.Query(authorization.IntegrationPointInstance)
#		query.ancestor(_get_project(uctx))
#		query.filter('user =',uctx.user)
#		query.order('type')
#		return platform.util.query_to_dict_list(query,50)


##############################################################
############    DASHBOARD BUSINESS
######

'''
this sets up the widget module so that it can be
a factory of widgets defined where we specify'''
widget.WIDGETS_MODULE = widgets

''' widget instance state constants '''
WIDGET_STATE_INIT 							= 0
WIDGET_STATE_REQUIRES_AUTH 					= 1
WIDGET_STATE_REQUIRES_SERVICE_SELECTION 	= 2
WIDGET_STATE_REQUIRES_DATA_PROXY 			= 21
WIDGET_STATE_READY 							= 3
WIDGET_SERVICE_STATE_OK 					= 4
WIDGET_SERVICE_STATE_FAILED_500 			= 500 
WIDGET_SERVICE_STATE_FAILED_NEED_AUTH 		= 501
WIDGET_SERVICE_STATE_FAILED_NEED_SELECTION 	= 502
WIDGET_SERVICE_STATE_FAILED_DATA_ERROR 		= 503
WIDGET_ERROR_NO_DEF						 	= 900

WIDGET_STATE_CORRELATION_REQUIRES_MEMBERS	= 620
WIDGET_STATE_CORRELATION_READY				= 621
WIDGET_STATE_CORRELATION_OK					= 622
WIDGET_STATE_CORRELATION_ERROR				= 623
WIDGET_STATE_CORRELATION_FAILED_500		    = 624

'''
Widget instances move through state transitions based
on user interaction as well as network events.
The constants above represent all the distinct states
the widgets moves through. A diagram of the primary
states can be viewed at:
http://code.google.com/p/impact-dashboard/wiki/WidgetInstances
'''
def enter_widget_state_machine(wi):
	state = wi.state

	if state == WIDGET_ERROR_NO_DEF:
		return wi.save(wi.user)

	elif state 	== WIDGET_STATE_INIT:

		if not wi.has_definition():
			wi.state = WIDGET_ERROR_NO_DEF
			return enter_widget_state_machine(wi)
		if wi.is_correlation():
			wi.state = WIDGET_STATE_CORRELATION_REQUIRES_MEMBERS
		else:
			wi.state = WIDGET_STATE_REQUIRES_AUTH
		return enter_widget_state_machine(wi)


	elif state 	== WIDGET_STATE_REQUIRES_AUTH:
			for connection_descriptor in wi.get_required_connections():
				if connection_descriptor['name'] not in wi.connections:
					connection = get_connection_for_user_by_type(wi.dashboard.project, wi.user, connection_descriptor['type'])
					if connection is None:
						wi.state = WIDGET_STATE_REQUIRES_AUTH
						return wi.save(wi.user)
					else:
						wi.connections[connection_descriptor['name']] = connection
			wi.state = WIDGET_STATE_REQUIRES_SERVICE_SELECTION
			return enter_widget_state_machine(wi)

	elif state 	== WIDGET_STATE_REQUIRES_SERVICE_SELECTION:
		try:
			sel_names = wi.get_service_selector_names()
			sel_types = wi.get_service_selector_types()
			for i in range(len(sel_names)):
				name = sel_names[i]
				type = WidgetDefinition.SELECTOR_TYPE_SELECT
				if len(sel_types)>i:
					type = sel_types[i]
				selector_display, selector_values  = wi.get_selector_data(name)
				if type == WidgetDefinition.SELECTOR_TYPE_SELECT and (not (name in wi.selector_values) or wi.selector_values[name] not in selector_values):
#auto selection removed at jonathan's request 7/8/2011
#					if len(selector_values) > 0:
#						wi.selector_values[name] = selector_values[0] # in the case where there is only one selector value set it and continue. the drop down will not fire a change if there is onlyone element in it. that is what lead me to this
#					else:
					return wi.save(wi.user)	#stay in state until they have selected all required selectors
				if type == WidgetDefinition.SELECTOR_TYPE_FREE_TEXT and (not (name in wi.selector_values) or wi.selector_values[name] is None or wi.selector_values[name] == ''):
					return wi.save(wi.user)	
			wi.state = WIDGET_STATE_REQUIRES_DATA_PROXY
			return enter_widget_state_machine(wi)
				#need toi deal with situation when someone attaches an analytics account with no data
				#i think this might happen in  auth though. might even be a different state
		except ImpactException, e:
			if e.code == 500:
				wi.state = WIDGET_SERVICE_STATE_FAILED_500
				return enter_widget_state_machine(wi)
			#this might be too primitive..need more insight into failures on 200
			wi.state = WIDGET_STATE_REQUIRES_AUTH
			return enter_widget_state_machine(wi)
	elif state == WIDGET_STATE_REQUIRES_DATA_PROXY:
		con = get_first_connection_from_widget_instance(wi)
		if con == None:
			pass#dont need to deal with data proxy if there is no connection or if it is a correlation
		else:
			proxy_key = build_proxy_key(wi)
			proxy     = WidgetDataProxy.get_or_insert(proxy_key,
											  parent=con,
											  connection = con,
											  widget=wi.widget,
	 										  selector_values=wi.selector_values,
	 										  selector_display_values=wi.selector_display_values)
			wi.proxy = proxy;

		wi.state = WIDGET_STATE_READY
		return enter_widget_state_machine(wi)
	elif state == WIDGET_STATE_READY:
			try:
				create_or_refresh_widget_data(wi)
				wi.state = WIDGET_SERVICE_STATE_OK
				return enter_widget_state_machine(wi)
			except ImpactException, e:
				if e.code == 500:
					wi.state = WIDGET_SERVICE_STATE_FAILED_500
					return enter_widget_state_machine(wi)
				#this might be too primitive..need more insight into failures on 200
				#need more robust error handling here for 400 Bad Request and others
				#this needs auth should just be for 401
				#need to make a state for client too like check request
			wi.state = WIDGET_STATE_REQUIRES_AUTH

			return enter_widget_state_machine(wi)
	elif state == WIDGET_SERVICE_STATE_OK:
		create_or_refresh_widget_data(wi) #TODO verify w. topher
		return wi.save(wi.user)
	elif state == WIDGET_SERVICE_STATE_FAILED_500:
		return wi.save(wi.user)
	elif state == WIDGET_STATE_CORRELATION_REQUIRES_MEMBERS:
		if len(wi.children) < 2:
			return wi.save(wi.user);
		else:
			wi.state = WIDGET_STATE_CORRELATION_READY
			return enter_widget_state_machine(wi)
	elif state == WIDGET_STATE_CORRELATION_READY:
			try:
				create_or_refresh_widget_data(wi)
				wi.state = WIDGET_STATE_CORRELATION_OK
				return enter_widget_state_machine(wi)
			except ImpactException, e:
				if e.code == 500:
					wi.state = WIDGET_STATE_CORRELATION_FAILED_500
					return enter_widget_state_machine(wi)
				#this might be too primitive..need more insight into failures on 200
				#need more robust error handling here for 400 Bad Request and others
				#this needs auth should just be for 401
				#need to make a state for client too like check request
				wi.state = WIDGET_STATE_CORRELATION_ERROR
				return enter_widget_state_machine(wi)

	elif state == WIDGET_STATE_CORRELATION_OK:
		return wi.save(wi.user)
	elif state == WIDGET_STATE_CORRELATION_FAILED_500:
		return wi.save(wi.user)
	elif state == WIDGET_STATE_CORRELATION_ERROR:
		return wi.save(wi.user)

def build_proxy_key(wi):
	proxy_key = [];
	for connection_descriptor in wi.get_required_connections():
		conn_instance = wi.connections[connection_descriptor['name']]
		proxy_key.append(conn_instance.type)
		proxy_key.append(str(conn_instance.uid))
		break #only take first one since we are now assuming widgets will only ever deal with one connection

	proxy_key.append(wi.definition().get_name())

	for name in wi.get_service_selector_names():
		proxy_key.append(name)
		proxy_key.append(str(wi.selector_values[name]))


	return '|'.join(proxy_key)


def get_data_proxy(wi):
	proxy_key = build_proxy_key(wi);
	return WidgetDataProxy.get_by_key_name(proxy_key,parent=wi.dashboard.project)

def get_first_connection_from_widget_instance(wi):
	for k,v in wi.connections.items():
		return v
	return None

def create_widget_instance(dash,wreg,rect):

	wi = WidgetInstance(parent=dash,
								widget=wreg,
								rect=rect,
								state=WIDGET_STATE_INIT,
								user=dash.user,
								selector_values={},
								selector_display_values={},
								connections={},
								props={})

	wi.props = wi.definition()._get_default_props()
	wi.save(dash.user)
	return wi

def create_correlation_widget_instance(dash,wreg,rect):

	wi = WidgetInstance(parent=dash,
								widget=wreg,
								rect=rect,
								state=WIDGET_STATE_CORRELATION_REQUIRES_MEMBERS,
								user=dash.user,
								selector_values={},
								selector_display_values={},
								connections={},
								props={})

	wi.props = wi.definition()._get_default_props()
	wi.save(dash.user)
	return wi

def add_connection(wi,connection):
	if connection is not None:
		wi.connection = connection

def get_connection_for_user_by_type(project,user,type):
	cs = get_connections_for_user_by_type(project, user, type, 1)
	if cs is None or len(cs)==0:
		return None
	return cs[0]

def get_connections_for_user_by_type(project,user,type,limit):
	query = db.Query(authorization.IntegrationPointInstance)
	query.ancestor(project)
	#query.filter('user =',user) #TODO do we want to do this?
	query.filter('type =',type)
	query.order("-modified")
	return query.fetch(limit)


#cron stuff
def process_all_widget_instances():
	#need to make sure temp projects are cleaned up#
	queue.page_apply(Project,process_projects,callback_page_size=20,page_size=20)


def process_projects(entities,params):
	projects = entities
	for project in projects:
		queue.page_apply(authorization.IntegrationPointInstance,process_project_connections,ancestor=project,callback_page_size=20,page_size=20)
		queue.page_apply(Dashboard,process_dashboard_proxyless_widget_instances,ancestor=project,callback_page_size=20,page_size=20)
	return None,None

def process_project_connections(entities,params):
	connections = entities
	for connection in connections:
		queue.page_apply(WidgetDataProxy,process_connection_data_proxies,ancestor=connection,callback_page_size=1,page_size=20)
	return None,None

def process_connection_data_proxies(entities,params):
	data_proxies = db.get(entities)
	for proxy in data_proxies:
		create_or_refresh_widget_data_proxy(proxy)
	return None,None


#proxyless

def process_dashboard_proxyless_widget_instances(entities,params):
	dashboards = entities
	for dashboard in dashboards:
		queue.page_apply(WidgetInstance,process_proxyless_widget_instances,ancestor=dashboard,filters=[['type = ', WIDGET_TYPE_NORMAL],['proxy = ',None]],callback_page_size=1,page_size=20)
		queue.page_apply(WidgetInstance,process_proxyless_widget_instances,ancestor=dashboard,filters=[['type = ', WIDGET_TYPE_CORRELATION_MEMBER],['proxy = ',None]],callback_page_size=1,page_size=20)
	return None,None
def process_proxyless_widget_instances(entities,params):
	entities = db.get(entities)
	for wi in entities:
		create_or_refresh_widget_data(wi)
	return None,None



#def process_dashboard_widget_instances(entities,params):
#	dashboards = entities
#	for dashboard in dashboards:
#		queue.page_apply(WidgetInstance,process_widget_instances,ancestor=dashboard,filters=[['type = ', WIDGET_TYPE_NORMAL]],callback_page_size=1,page_size=20)
#	return None,None
#def process_widget_instances(entities,params):
#	entities = db.get(entities)
#	for wi in entities:
#		create_or_refresh_widget_data(wi)
#	return None,None



#TODO revamp widget data collection so that it operates on the definition/connection level
#PSEUDO CODE
#	for user in users.all()
#		for connect in user.connection
#			for def in get_defs(connect.type)
#				for selector_name in def.get_selector_names()
#					for selector_option in def.get_selector_data(selector_name)
#						selector_values[selector_name] = selector_option
#						data = def.get_data(selector_values,{}) #props is empty because this is not for an instance
#						w = WidgetData(parent=project, date, def, connect, selector_name, selector_option, data)
#						w.put()
#		for wi of get_no_connection_widget_instances(user)
#			w = WidgetData(parent=wi,date, def, None, None, None, wi.get_data())
#			w.put()

def create_or_refresh_widget_data(wi,date=None):


	if not wi.has_definition():
		raise ImpactException('No definition for widget ['+wi.widget.key().name()+']')

	if wi.proxy != None:
		return create_or_refresh_widget_data_proxy(wi.proxy)


	try:
		data = wi.get_data()
	except:
		logging.info('can t get data for '+str(wi))
		traceback.print_exc()
		return
	if data is None:
		return

	now = datetime.datetime.utcnow()
	#year,month,day,hour,minute,second,weekday,day_of_year,dst = now.timetuple()
	#logging.info('DATE WAS YEAR '+str(year)+' MONTH '+str(month)+' day '+str(day)+' hour '+str(hour)+' min '+str(minute)+' sec '+str(second)+' doy '+str(day_of_year))
	date,date_key = platform.util.date_floor(now, config['data_quantize_unit'], config['data_quantize_interval'])
	#year,month,day,hour,minute,second,weekday,day_of_year,dst = date.timetuple()
	#logging.info('DATE IS NOW YEAR '+str(year)+' MONTH '+str(month)+' day '+str(day)+' hour '+str(hour)+' min '+str(minute)+' sec '+str(second)+' doy '+str(day_of_year))

	wd = WidgetData.get_by_key_name(date_key, wi)

	if wd is None:
		wd = WidgetData(parent=wi,key_name=date_key,data=data,date=date)
	else:
		wd.data = data

	wd.save(wi.user)
	return wd

def create_or_refresh_widget_data_proxy(proxy,date=None):

	try:
		data = proxy.get_data()
	except:
		logging.info('can t get data for '+str(proxy))
		traceback.print_exc()
		return
	if data is None:
		return

	now = datetime.datetime.utcnow()
	#year,month,day,hour,minute,second,weekday,day_of_year,dst = now.timetuple()
	#logging.info('DATE WAS YEAR '+str(year)+' MONTH '+str(month)+' day '+str(day)+' hour '+str(hour)+' min '+str(minute)+' sec '+str(second)+' doy '+str(day_of_year))
	date,date_key = platform.util.date_floor(now, config['data_quantize_unit'], config['data_quantize_interval'])
	#year,month,day,hour,minute,second,weekday,day_of_year,dst = date.timetuple()
	#logging.info('DATE IS NOW YEAR '+str(year)+' MONTH '+str(month)+' day '+str(day)+' hour '+str(hour)+' min '+str(minute)+' sec '+str(second)+' doy '+str(day_of_year))

	wd = WidgetData.get_by_key_name(date_key, proxy)

	if wd is None:
		wd = WidgetData(parent=proxy,key_name=date_key,data=data,date=date)
	else:
		wd.data = data

	wd.save(proxy.user)
	proxy.save() #to update the mod time, ie last data collected at
	return wd


def get_data_for_trend_props_widget(wi):
	#display-style total || trend
	#trend-type last_2 || last_5 || last_10
	#show-events 0 || 1
	ret = {}
	display_style,trend_type,show_events,num_data_points = get_trend_prop_values(wi.props)
	results = get_widget_instance_data(wi, num_data_points)
	values = []
	for result in results:
		values.append(result.to_dict())
	ret['values'] = values

	return ret

def get_data_for_trend_props_correlation_widget(wi):
	ret_dict = {'corr':[]}
	display_style,trend_type,show_events,num_data_points = get_trend_prop_values(wi.props)
	cc = db.get(wi.children)
	for c in cc:
		results = get_widget_instance_data(c, num_data_points)
		values = []
		for result in results:
			values.append(result.to_dict())
		ret_dict['corr'].append({'name':c.widget.name,'values':values})
	return ret_dict
	

def get_trend_prop_values(props):
	display_style  = props.get('display-style') or 'total'
	trend_type     = props.get('time-span') or '7'
	show_events    = props.get('show-events') or '0'
	#default
	num_data_points = 1;
	if display_style == 'trend' or display_style == 'percent-change':
		num_data_points = int(trend_type)
	return (display_style,trend_type,show_events,num_data_points)

def get_widget_instance_data(wi,num_data_points):
	query = db.Query(WidgetData)
	if wi.proxy:
		query.ancestor(wi.proxy)
	else:
		query.ancestor(wi)
	##TODO new data collection system requires something like:
	##	if requires_conn(wi) ancestor is project
	##	else ancestor is wi
	##TODO ...use date range w/ dates, not num_data_points
	##query.filter('date <= ', now)
	##query.filter('date >', now-num_data_points*TIME_SLICE)
	##TODO ...and
	##if requires_conn:
	##   query.filter('def = ',wi.def)
	##   query.filter('conn = ',wi.conn)
	##   sel_name = wi.def.selector_name[0]    ## should loop for all
	##   query.filter('selector_name=',sel_name)
	##   query.filter('selector_val=', wi.selector_values[sel_name]
	query.order('-date')
	results = query.fetch(num_data_points)
	return results



###EVENTS







# CONNECTIONS

class ConnectionsPage(platform.TemplateHandler):

	def __init__(self):
		self.template = "templates/connections.html"

	def prepare_page(self,uctx,inputs,pagedata):
		pagedata['project'] = _get_project(uctx)


class ConnectionsApi(platform.ServiceHandler):

	def list(self,uctx,args,params):
		"""returns a list of user's projects"""
		query = db.Query(IntegrationPointInstance)
		query.ancestor(_get_project(uctx))
		ips = query.fetch(100)
		if len(args)==0:
			return platform.util.query_to_dict_list(ips)
		ret = []
		for ip in ips:
			query = db.Query(WidgetDataProxy)
			query.ancestor(ip)
			ip_dict = ip.to_dict()
			ip_dict['children'] = []
			wdps = query.fetch(1000)
			for wdp in wdps:
				wpd_dict = wdp.to_dict()
				wpd_dict['connection'] = None
				ip_dict['children'].append(wpd_dict)
			ret.append(ip_dict)
		return ret

	def delete(self,uctx,args,params):
		"""removes an event. requires the ip key."""
		if len(args)!=1:
			raise Exception('Requires ip key.')
		ip = IntegrationPointInstance.get(db.Key(encoded=args[0]))
		if ip == None or ip.user != uctx.user:
			raise Exception('There is no such ip')
		query = db.Query(WidgetDataProxy)
		query.ancestor(ip)
		wdps = query.fetch(1000)
		for wdp in wdps:
			query = db.Query(WidgetInstance)
			query.filter('proxy =', wdp)
			wis = query.fetch(1000)
			for wi in wis:
				wi.delete()
			wdp.delete()
		ip.delete();
		return ip.to_dict()



class ConnectionsAuthPage(platform.ImpactBaseHandler):
	'''
	When the UI needs a Connection, it interacts with the ConnectionsAuthPage.
	It differs from DashboardAuthPage in that it is contextless.
	This page must be requested in a pop up window.
	'''
	def process(self,uctx,path):
		type 	= self.request.get('type')
		callback 	= self.request.get('callback')
		return_to = '/connections/authorize/return/'+callback
		redirect_url = '/authorize/'+type+'?r='+return_to+'&p='+str(_get_project(uctx).key())
		self.redirect(redirect_url)

class ConnectionsAuthReturnPage(platform.ImpactBaseHandler):

	def process(self,uctx,path):
		args = path.split('/')
		callback = 'authorization_complete'
		if (len(args)!=0):
			callback = args[0]
		ip = authorization.IntegrationPointInstance.get(db.Key(encoded=self.request.get("ip")))
		self.response.out.write('<script>window.opener.'+callback+'('+platform.util.entity_to_json(ip)+');this.close()</script>')


##################################
#### UTILITIES

SESSION_PROJECT_CTX_KEY = 'project_ctx'
def _get_project(uctx):
	"""this returns the project context for a user"""
	if uctx.has_prop(SESSION_PROJECT_CTX_KEY):
		project = uctx.get_prop(SESSION_PROJECT_CTX_KEY)
	elif uctx.user is None:
		return None
	else:
		query = db.Query(Project)
		query.filter('user =',uctx.user)
		project = query.fetch(1)[0]
		_set_project(uctx, project)
	return project

def _set_project(uctx,project):
	uctx.set_prop(SESSION_PROJECT_CTX_KEY, project)

def _get_dashboard_by_name(uctx,name):
	query = db.Query(Dashboard)
	query.filter('user =',uctx.user)
	query.filter('name =',name)
	return query.fetch(1)

def _dashboard_name_exists(uctx,name):
	return len(_get_dashboard_by_name(uctx, name))!=0


def _create_project(user, project_name='Default Project', dashboard_name='Master Dashboard'):
	#def txn():
		project = Project(name=project_name,parent=user)
		project.save(user)
		dshbrd = Dashboard(name=dashboard_name, parent=project)
		dshbrd.save(user)
		return project
	#return db.run_in_transaction(txn)


#TODO rename & move to platform
#deletes all C (models) with ancestor p
#used in delete of Project , Dashboard & WidgetInstance
def _del(C,p):
	queue.page_apply(C,queue.page_apply_delete, ancestor=p,callback_page_size=2, page_size=48,multiplex=False)


##################################
#### GUARDS

def guard_permissions(uctx,project):
	# if you are not the owner
	if project.user != uctx.user:
		raise ImpactException('Permissions error')


##################################
#### ENTITIES
#need to be running this in transactions..the deletes and chowns



class Project(platform.ImpactModel):
	name = db.StringProperty()

	def delete(self):
		_del(Dashboard,self);
		platform.ImpactModel.delete(self)

class Dashboard(platform.ImpactModel):
	# parent is a Project
	name = db.StringProperty()
	description = db.TextProperty()
	tags = db.ListProperty(str)
	public = db.BooleanProperty(default=True)

	def delete(self):
		_del(WidgetInstance,self)
		platform.ImpactModel.delete(self)

	@property
	def project(self):
		return self.parent()

	def get_widgets_query(self,keys_only=True):
		return WidgetInstance.all(keys_only=keys_only).ancestor(self).filter('type = ', WIDGET_TYPE_NORMAL)

	@property
	def widgets(self):
		return self.get_widgets_query(keys_only=False).fetch(512) #only return first 512 widgets in dashboard. ha ha.

WIDGET_TYPE_NORMAL 				= 0x00
WIDGET_TYPE_CORRELATION_MEMBER 	= 0x01

class WidgetDataProxy(platform.ImpactModel):
	#!!!parent is connection thinks
	type 				= db.IntegerProperty()
	state 				= db.IntegerProperty()
	selector_values 	= platform.PickleProperty()
	selector_display_values 	= platform.PickleProperty()
	widget 				= db.ReferenceProperty(widget.WidgetRegistry)
	connection 		    = db.ReferenceProperty(authorization.IntegrationPointInstance)

	def get_data(self):
		return self.widget.definition()._get_data(self.connection,self.selector_values,{})

	def delete(self):
		_del(WidgetData,self)

class WidgetInstance(platform.ImpactModel):
	# parent is a Dashboard
	state 				= db.IntegerProperty()
	name 				= db.StringProperty()
	widget 				= db.ReferenceProperty(widget.WidgetRegistry)
	rect 				= db.ListProperty(int)
	props 				= platform.PickleProperty()
	selector_values 	= platform.PickleProperty()
	selector_display_values 	= platform.PickleProperty()
	connections			= platform.PickleProperty()
	type				= db.IntegerProperty(default=WIDGET_TYPE_NORMAL)
	children			= db.ListProperty(db.Key, default=None)
	proxy 				= db.ReferenceProperty(WidgetDataProxy)

	@property
	def definition(self):
		return self.widget.definition

	def has_definition(self):
		return self.widget.has_definition()

	@property
	def dashboard(self):
		return self.parent()

	def delete(self):
		_del(WidgetData,self)
		if len(self.children) != 0:
			db.delete(self.children)
			#self.expand('children')
			#for c in self.children:
			#	c.delete()
		platform.ImpactModel.delete(self)
		#self.children = []

	def clone(self,p):
		return WidgetInstance(parent	= p,
					state 				= self.state,
					name 				= self.name,
					widget 				= self.widget.key(),
					rect 				= self.rect,
					props 				= self.props,
					selector_values 	= self.selector_values,
					selector_display_values 	= self.selector_display_values,
					connections			= self.connections,
					proxy				= self.proxy,
					connection 			= self.connection)


	#could look into using meta classes for this
	#also definition can be cached in widget registry class
	#instead of creating a new one every time
	def get_data(self):
		#return self.definition()._get_data(self)
		if self.proxy:
			return self.definition()._get_data(self.proxy.connection,self.proxy.selector_values,self.props)
		else:
			return self.definition()._get_data(None,self.selector_values,self.props)

	def get_required_connections(self):
		return self.definition()._get_connections()

	def get_service_selector_names(self):
		return self.definition()._get_service_selector_names()

	def get_service_selector_types(self):
		return self.definition()._get_service_selector_types()

	def get_selector_data(self,name):
		return self.definition()._get_selector_data(self,name)

	def get_props_editor(self):
		return self.definition()._get_props_editor()

	def get_props_editor_template(self):
		return self.definition()._get_props_editor_template()

	def is_correlation(self):
		return self.definition()._is_correlation()

	def is_live_data_widget(self):
		return self.definition()._is_live_data_widget()

	def to_dict(self):
		self.expand('children')
		return platform.ImpactModel.to_dict(self)


class WidgetData(platform.ImpactModel):
	#parent is widget instance... for now....
	date = db.DateTimeProperty()
	data = platform.PickleProperty()





